"""
steam游戏数据爬取
天津工业大学 大数据实训 第四组
杨玺锟
2020-1-8
"""

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

from kafka import KafkaProducer
from scrapy.utils.serialize import ScrapyJSONEncoder


class SteamSpiderPipeline(object):
    def __init__(self):
        # kafka_ip_port = "192.168.2.10:9092"
        # self.client = KafkaClient(hosts=kafka_ip_port)
        # self.producer = self.client.topics["test".encode(encoding="UTF-8")].get_producer()

        self.producer = KafkaProducer(bootstrap_servers=['192.168.2.10:9092'])

    def close_spider(self, spider):
        self.producer.close()

    def process_item(self, item, spider):
        game = {
            'game_name': item["game_name"],
            'game_price': item["game_price"],
            'price_discount': item["price_discount"],
            'release_date': item["release_date"],
            'game_review_count': item["game_review_count"],
            'positive_review_ratio': item["positive_review_ratio"],
            'game_type': item["game_type"],
            'game_about': item["game_about"],
            'img_src': item["img_src"]
        }
        encoder = ScrapyJSONEncoder()
        encoded_tmp = encoder.encode(game).encode()  # 字典转化为json串之后再进行二进制编码(序列化)
        self.producer.send('test', encoded_tmp)
        return item
